diff --git a/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml b/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml index 53a7f36..58083f8 100644 --- a/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml +++ b/mlperf_logging/compliance_checker/training_6.1.0/closed_dlrmv4.yaml @@ -5,12 +5,17 @@ 'eval_interval_samples': None, 'first_eval_samples': None, 'opt_base_learning_rate': None, + 'lr_min': None, + 'lr_max': None, 'train_samples': 2290835423, 'target_accuracy': 0.75, }) # Only the target learning rate and the global batch size may be tuned; every -# other hyperparameter below is fixed by the closed division. +# other hyperparameter below is fixed by the closed division. The learning rate +# is further confined to a band around the linear scaling law the reference +# sweep was collected on, 1e-6 * global_batch_size / 8192, since convergence is +# only comparable against the RCPs near that recipe. - KEY: NAME: global_batch_size @@ -38,6 +43,9 @@ math.floor((4480 * v['value'] + 135331840) / 3) / s['eval_interval_samples'] )) * s['eval_interval_samples'] + # Half to 1.5x the reference learning rate for this batch size. + s['lr_min'] = 0.5e-6 * v['value'] / 8192 + s['lr_max'] = 1.5e-6 * v['value'] / 8192 - KEY: NAME: train_samples @@ -64,7 +72,10 @@ - KEY: NAME: opt_base_learning_rate REQ: EXACTLY_ONE - CHECK: " v['value'] > 0 " + # The bounds are derived from global_batch_size, and the rules run in log + # order, so a log that reports the learning rate first fails the guard + # rather than raising on the comparison. + CHECK: " s['lr_min'] is not None and s['lr_min'] * (1 - 1e-9) <= v['value'] <= s['lr_max'] * (1 + 1e-9) " POST: " s['opt_base_learning_rate'] = v['value'] " - KEY: diff --git a/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml b/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml index cbf79a9..98d3b0c 100644 --- a/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml +++ b/mlperf_logging/compliance_checker/training_6.1.0/open_dlrmv4.yaml @@ -1,7 +1,14 @@ +# The open division is for algorithmic changes, not for buying convergence with +# a learning rate the closed division would reject, so the same band around the +# reference scaling law applies here. + - KEY: NAME: global_batch_size REQ: EXACTLY_ONE CHECK: " is_integer(v['value']) and v['value'] > 0 " + POST: | + s['lr_min'] = 0.5e-6 * v['value'] / 8192 + s['lr_max'] = 1.5e-6 * v['value'] / 8192 - KEY: NAME: gradient_accumulation_steps @@ -30,6 +37,10 @@ - KEY: NAME: opt_base_learning_rate REQ: EXACTLY_ONE + # The bounds come from global_batch_size, and the rules run in log order, so + # a log that reports the learning rate first fails the guard rather than + # raising on the comparison. + CHECK: " s.get('lr_min') is not None and s['lr_min'] * (1 - 1e-9) <= v['value'] <= s['lr_max'] * (1 + 1e-9) " - KEY: NAME: opt_adam_beta_1 @@ -58,6 +69,9 @@ - KEY: NAME: opt_sparse_base_learning_rate REQ: EXACTLY_ONE + # Unlike the closed division this is not pinned to the dense rate, so it + # needs the band applied independently. + CHECK: " s.get('lr_min') is not None and s['lr_min'] * (1 - 1e-9) <= v['value'] <= s['lr_max'] * (1 + 1e-9) " - KEY: NAME: opt_learning_rate_warmup_steps