diff --git a/mlperf_logging/benchmark_meta.py b/mlperf_logging/benchmark_meta.py index 3304b88..81ff3da 100644 --- a/mlperf_logging/benchmark_meta.py +++ b/mlperf_logging/benchmark_meta.py @@ -25,6 +25,7 @@ 'flux1': 10, 'gpt_oss_20b': 10, 'deepseekv3_671b': 3, + 'qwen35_397b_grpo': 3, }, 'hpc' : { @@ -166,6 +167,14 @@ 'llama31_405b', 'gpt_oss_20b', 'deepseekv3_671b' + ], + '6.1': [ + 'llama31_8b', + 'flux1', + 'llama2_70b_lora', + 'gpt_oss_20b', + 'deepseekv3_671b', + 'qwen35_397b_grpo' ] }, diff --git a/mlperf_logging/compliance_checker/README.md b/mlperf_logging/compliance_checker/README.md index 19166ef..8e8aea2 100644 --- a/mlperf_logging/compliance_checker/README.md +++ b/mlperf_logging/compliance_checker/README.md @@ -31,6 +31,7 @@ As log examples use [NVIDIA's training logs](https://github.com/mlperf/training_ 6.0.0/closed_llama2_70b_lora.yaml 6.0.0/closed_flux1.yaml 6.0.0/closed_gpt_oss_20b.yaml + 6.1.0/closed_qwen35_397b_grpo.yaml 6.0.0/open_llama31_8b.yaml 6.0.0/open_llama31_405b.yaml 6.0.0/open_dlrm_dcnv2.yaml diff --git a/mlperf_logging/compliance_checker/mlp_compliance.py b/mlperf_logging/compliance_checker/mlp_compliance.py index 3d2b30f..09b9152 100644 --- a/mlperf_logging/compliance_checker/mlp_compliance.py +++ b/mlperf_logging/compliance_checker/mlp_compliance.py @@ -326,7 +326,7 @@ def get_parser(): parser.add_argument('--usage', type=str, default='training', choices=usage_choices(), help='what WG do the benchmarks come from') - parser.add_argument('--ruleset', type=str, default='6.0.0', + parser.add_argument('--ruleset', type=str, default='6.1.0', choices=rule_choices(), help='what version of rules to check the log against') parser.add_argument('--config', type=str, diff --git a/mlperf_logging/compliance_checker/training_6.1.0/closed_common.yaml b/mlperf_logging/compliance_checker/training_6.1.0/closed_common.yaml index 04f00f3..8920d83 100755 --- a/mlperf_logging/compliance_checker/training_6.1.0/closed_common.yaml +++ b/mlperf_logging/compliance_checker/training_6.1.0/closed_common.yaml @@ -2,7 +2,7 @@ - KEY: NAME: submission_benchmark REQ: EXACTLY_ONE - CHECK: " v['value'] in ['flux1', 'llama31_8b', 'llama2_70b_lora', 'gpt_oss_20b', 'deepseekv3_671b'] " + CHECK: " v['value'] in ['flux1', 'llama31_8b', 'llama2_70b_lora', 'gpt_oss_20b', 'deepseekv3_671b', 'qwen35_397b_grpo'] " POST: " enqueue_config('training_6.1.0/closed_{}.yaml'.format(v['value'])) " - KEY: diff --git a/mlperf_logging/compliance_checker/training_6.1.0/closed_qwen35_397b_grpo.yaml b/mlperf_logging/compliance_checker/training_6.1.0/closed_qwen35_397b_grpo.yaml new file mode 100644 index 0000000..f53e6a1 --- /dev/null +++ b/mlperf_logging/compliance_checker/training_6.1.0/closed_qwen35_397b_grpo.yaml @@ -0,0 +1,128 @@ +- BEGIN: + CODE: > + s.update({ + 'global_batch_size': None, + 'num_prompts_per_step': None, + 'next_validation_samples': None, + 'validation_start_samples': None, + 'target_accuracy': 0.69, + }) + +- KEY: + NAME: global_batch_size + REQ: EXACTLY_ONE + CHECK: " is_integer(v['value']) and v['value'] > 0 " + POST: | + s['global_batch_size'] = v['value'] + s['learning_rate'] = 1.0e-6 * (s['global_batch_size'] / 256) ** 0.5 + s['validation_start_samples'] = v['value'] * math.ceil(2.5 + 3840 / v['value']) + s['next_validation_samples'] = s['validation_start_samples'] + +- KEY: + NAME: max_sequence_length + REQ: EXACTLY_ONE + CHECK: " v['value'] == 65536 " + +- KEY: + NAME: eval_samples + REQ: EXACTLY_ONE + CHECK: " v['value'] == 256 " + +- KEY: + NAME: init_checkpoint_step + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0 " + +- KEY: + NAME: max_steps + REQ: EXACTLY_ONE + +- KEY: + NAME: opt_name + REQ: EXACTLY_ONE + CHECK: " v['value'] == 'adamw' " + +- KEY: + NAME: opt_base_learning_rate + REQ: EXACTLY_ONE + CHECK: " abs(v['value'] - s['learning_rate']) < 1e-9 " + +- KEY: + NAME: opt_adamw_beta_1 + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0.9 " + +- KEY: + NAME: opt_adamw_beta_2 + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0.999 " + +- KEY: + NAME: opt_adamw_epsilon + REQ: EXACTLY_ONE + CHECK: " v['value'] == 1e-8 " + +- KEY: + NAME: opt_adamw_weight_decay + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0.0 " + +- KEY: + NAME: opt_gradient_clip_norm + REQ: EXACTLY_ONE + CHECK: " abs(v['value'] - 0.125 * (256 / s['global_batch_size']) ** 0.5) < 1e-9 " + +- KEY: + NAME: opt_learning_rate_warmup_steps + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0 " + +- KEY: + NAME: opt_learning_rate_decay_schedule + REQ: EXACTLY_ONE + CHECK: " v['value'] == 'constant' " + +- KEY: + NAME: generation_training_rollout_temperature + REQ: EXACTLY_ONE + CHECK: " v['value'] == 1.0 " + +- KEY: + NAME: generation_training_rollout_top_p + REQ: EXACTLY_ONE + CHECK: " v['value'] == 1.0 " + +- KEY: + NAME: generation_validation_rollout_temperature + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0.1 " + +- KEY: + NAME: generation_validation_rollout_top_p + REQ: EXACTLY_ONE + CHECK: " v['value'] == 0.95 " + +- KEY: + NAME: num_prompts_per_step + REQ: EXACTLY_ONE + CHECK: " is_integer(v['value']) and v['value'] > 0 " + POST: " s['num_prompts_per_step'] = v['value'] " + +- KEY: + NAME: num_generations_per_prompt + REQ: EXACTLY_ONE + CHECK: + - "v['value'] == 16" + - "s['global_batch_size'] == s['num_prompts_per_step'] * v['value']" + +- KEY: + NAME: eval_accuracy + REQ: AT_LEAST_ONE + CHECK: + - "'samples_count' in v['metadata']" + - "v['metadata']['samples_count'] == s['next_validation_samples']" + - "0.0 <= v['value'] <= 1.0" + POST: | + if v['metadata']['samples_count'] == s['next_validation_samples']: + s['next_validation_samples'] += s['global_batch_size'] + ATLEAST_ONE_CHECK: "v['value'] > s['target_accuracy']" diff --git a/mlperf_logging/rcp_checker/rcp_checker.py b/mlperf_logging/rcp_checker/rcp_checker.py index 5568d20..87ae14a 100644 --- a/mlperf_logging/rcp_checker/rcp_checker.py +++ b/mlperf_logging/rcp_checker/rcp_checker.py @@ -62,6 +62,7 @@ def is_version_at_least(version, min_version): 'llama31_8b': 10, 'gpt_oss_20b': 10, 'deepseekv3_671b': 3, + 'qwen35_397b_grpo': 3, }, "hpc": { 'cosmoflow': 10, @@ -111,7 +112,7 @@ def read_submission_file(result_file, ruleset, use_train_samples): eval_metric = json.loads(eval_accuracy_str)["metadata"]["metric"] eval_score = json.loads(eval_accuracy_str)["value"] stable_diffusion_eval_results[eval_step][eval_metric] = eval_score - elif benchmark in {"llama2_70b_lora", "flux1", "llama31_405b", "llama31_8b", "gpt_oss_20b", "deepseekv3_671b"} and ("eval_error" in str or "eval_accuracy" in str): + elif benchmark in {"llama2_70b_lora", "flux1", "llama31_405b", "llama31_8b", "gpt_oss_20b", "deepseekv3_671b", "qwen35_397b_grpo"} and ("eval_error" in str or "eval_accuracy" in str): eval_accuracy_str = str conv_epoch = json.loads(eval_accuracy_str)["metadata"]["samples_count"] eval_score = json.loads(eval_accuracy_str)["value"] diff --git a/mlperf_logging/rcp_checker/training_6.1.0/rcps_qwen35_397b_grpo.json b/mlperf_logging/rcp_checker/training_6.1.0/rcps_qwen35_397b_grpo.json new file mode 100644 index 0000000..5701aa9 --- /dev/null +++ b/mlperf_logging/rcp_checker/training_6.1.0/rcps_qwen35_397b_grpo.json @@ -0,0 +1,61 @@ +{ + "qwen35_397b_grpo_ref_256": + { + "Benchmark": "qwen35_397b_grpo", + "Creator": "NVIDIA", + "When": "Reference RCPs before 6.1 submission", + "Platform": "64 NVIDIA GB300 nodes", + "Precision": "BF16", + "BS": 256, + "Hyperparams": { + "opt_base_learning_rate": 1e-06, + "opt_gradient_clip_norm": 0.125, + "num_prompts_per_step": 16, + "num_generations_per_prompt": 16, + "gradient_accumulation_steps": 32 + }, + "Epochs to converge": [ + 4608, 4608, 4608, 4608, 4608, 4608 + ] + }, + + "qwen35_397b_grpo_ref_512": + { + "Benchmark": "qwen35_397b_grpo", + "Creator": "NVIDIA", + "When": "Reference RCPs before 6.1 submission", + "Platform": "64 NVIDIA GB300 nodes", + "Precision": "BF16", + "BS": 512, + "Hyperparams": { + "opt_base_learning_rate": 1.4142135624e-06, + "opt_gradient_clip_norm": 0.08838834765, + "num_prompts_per_step": 32, + "num_generations_per_prompt": 16, + "gradient_accumulation_steps": 64 + }, + "Epochs to converge": [ + 5120, 5120, 5120, 5120, 5120, 5120 + ] + }, + + "qwen35_397b_grpo_ref_1024": + { + "Benchmark": "qwen35_397b_grpo", + "Creator": "NVIDIA", + "When": "Reference RCPs before 6.1 submission", + "Platform": "64 NVIDIA GB300 nodes", + "Precision": "BF16", + "BS": 1024, + "Hyperparams": { + "opt_base_learning_rate": 2e-06, + "opt_gradient_clip_norm": 0.0625, + "num_prompts_per_step": 64, + "num_generations_per_prompt": 16, + "gradient_accumulation_steps": 128 + }, + "Epochs to converge": [ + 7168, 7168, 7168, 7168, 7168, 7168 + ] + } +}